{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "30b90bc6-25db-439d-810c-e1dfa9769aa2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-21T18:49:53.818778Z",
     "start_time": "2024-10-21T18:49:53.360388Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "请求成功!\n",
      "页面内容已保存到当前文件夹: 100095516761.html\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "url = \"https://item.jd.com/100095516761.html\"\n",
    "\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36\"\n",
    "}\n",
    "\n",
    "response = requests.get(url, headers=headers)\n",
    "\n",
    "if response.status_code == 200:\n",
    "    print(\"请求成功!\")\n",
    "    \n",
    "    # 从URL中提取文件名\n",
    "    filename = url.split(\"/\")[-1]\n",
    "    \n",
    "    # 确保文件有.html后缀\n",
    "    if not filename.endswith('.html'):\n",
    "        filename += '.html'\n",
    "    \n",
    "    # 保存响应内容到文件\n",
    "    with open(filename, 'w', encoding='utf-8') as f:\n",
    "        f.write(response.text)\n",
    "    \n",
    "    print(f\"页面内容已保存到当前文件夹: {filename}\")\n",
    "else:\n",
    "    print(\"请求失败,状态码:\", response.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe5c3a29-e15e-4cdc-91aa-58200b5e7371",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "请使用京东 App 扫描二维码登录\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "Interrupted by user",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 33\u001b[0m\n\u001b[0;32m     30\u001b[0m login_button\u001b[38;5;241m.\u001b[39mclick()\n\u001b[0;32m     32\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m请使用京东 App 扫描二维码登录\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 33\u001b[0m \u001b[38;5;28minput\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m登录完成后按回车键继续...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     35\u001b[0m \u001b[38;5;66;03m# 搜索产品\u001b[39;00m\n\u001b[0;32m     36\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m华为p70 pro\u001b[39m\u001b[38;5;124m'\u001b[39m\n",
      "File \u001b[1;32mD:\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py:1262\u001b[0m, in \u001b[0;36mKernel.raw_input\u001b[1;34m(self, prompt)\u001b[0m\n\u001b[0;32m   1260\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw_input was called, but this frontend does not support input requests.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1261\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m StdinNotImplementedError(msg)\n\u001b[1;32m-> 1262\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_input_request(\n\u001b[0;32m   1263\u001b[0m     \u001b[38;5;28mstr\u001b[39m(prompt),\n\u001b[0;32m   1264\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parent_ident[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshell\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[0;32m   1265\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_parent(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshell\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1266\u001b[0m     password\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m   1267\u001b[0m )\n",
      "File \u001b[1;32mD:\\anaconda3\\Lib\\site-packages\\ipykernel\\kernelbase.py:1305\u001b[0m, in \u001b[0;36mKernel._input_request\u001b[1;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[0;32m   1302\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[0;32m   1303\u001b[0m     \u001b[38;5;66;03m# re-raise KeyboardInterrupt, to truncate traceback\u001b[39;00m\n\u001b[0;32m   1304\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInterrupted by user\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m-> 1305\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1306\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[0;32m   1307\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid Message:\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from time import sleep\n",
    "from lxml import etree\n",
    "import pandas as pd\n",
    "\n",
    "# 初始化相关参数\n",
    "url = 'https://www.jd.com/'\n",
    "pages = 2  # 要抓取的产品页数\n",
    "comment_pages = 5  # 要抓取的评论页数\n",
    "\n",
    "# 启动 Chrome 浏览器\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "# 创建 WebDriverWait 对象\n",
    "wait = WebDriverWait(driver, 10)\n",
    "\n",
    "# 用于存储产品和评论的数据\n",
    "data = pd.DataFrame()\n",
    "comments = pd.DataFrame()\n",
    "\n",
    "# 打开京东首页\n",
    "driver.get(url)\n",
    "\n",
    "# 等待并点击登录按钮\n",
    "login_button = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'link-login')))\n",
    "login_button.click()\n",
    "\n",
    "print(\"请使用京东 App 扫描二维码登录\")\n",
    "input(\"登录完成后按回车键继续...\")\n",
    "\n",
    "# 搜索产品\n",
    "key = '华为p70 pro'\n",
    "search_box = wait.until(EC.presence_of_element_located((By.ID, 'key')))\n",
    "search_box.clear()\n",
    "search_box.send_keys(key)  # 输入搜索关键词\n",
    "search_box.send_keys(Keys.RETURN)  # 回车进行搜索\n",
    "sleep(3)\n",
    "\n",
    "# 定义函数以向下滚动页面\n",
    "def scroll_down():\n",
    "    for _ in range(2):\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        sleep(3)\n",
    "\n",
    "# 定义函数以提取页面上的产品信息\n",
    "def get_content():\n",
    "    global data  # 使用全局变量来存储产品数据\n",
    "    html = etree.HTML(driver.page_source)\n",
    "    items = html.xpath('//div[@class=\"gl-i-wrap\"]')\n",
    "    for item in items:\n",
    "        D = {}\n",
    "        # 提取价格\n",
    "        D['price'] = item.xpath('.//div[@class=\"p-price\"]/strong/i/text()')[0]\n",
    "        # 提取评论数\n",
    "        D['comment'] = item.xpath('.//div[@class=\"p-commit\"]/strong/a/text()')[0]\n",
    "        # 提取店铺名称\n",
    "        shopname = item.xpath('.//div[@class=\"p-shop\"]/span/a/text()')\n",
    "        D['shopname'] = shopname[0] if shopname else 'None'\n",
    "        # 提取产品链接\n",
    "        D['URL'] = 'https:' + item.xpath('.//div[@class=\"p-img\"]/a/@href')[0]\n",
    "        # 提取产品标题\n",
    "        title = item.xpath('.//div[@class=\"p-name p-name-type-2\"]/a/em')[0].xpath('string(.)').strip()\n",
    "        D['title'] = title\n",
    "        # 提取图片链接\n",
    "        image_url = item.xpath('.//div[@class=\"p-img\"]/a/img/@data-lazy-img')\n",
    "        D['pnglink'] = 'https:' + image_url[0] if image_url and image_url[0] != 'done' else 'https:' + item.xpath('.//div[@class=\"p-img\"]/a/img/@src')[0]\n",
    "        \n",
    "        # 将数据添加到 DataFrame 中\n",
    "        data = pd.concat([data, pd.DataFrame([D])])\n",
    "\n",
    "# 定义函数以点击下一页按钮\n",
    "def next_page():\n",
    "    try:\n",
    "        next_button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.pn-next:not(.disabled)')))\n",
    "        driver.execute_script(\"arguments[0].click();\", next_button)  # 点击下一页按钮\n",
    "        sleep(4)\n",
    "        return True\n",
    "    except:\n",
    "        print(\"No more pages or unable to click next page button\")\n",
    "        return False\n",
    "\n",
    "# 抓取指定页数的产品信息\n",
    "for _ in range(pages):\n",
    "    scroll_down()  # 向下滚动页面\n",
    "    get_content()  # 提取页面内容\n",
    "    if not next_page():  # 如果无法点击下一页按钮，停止抓取\n",
    "        break\n",
    "\n",
    "# 获取产品评论（针对第一页的第一个产品）\n",
    "if not data.empty:\n",
    "    first_product_url = data.iloc[0]['URL']\n",
    "    driver.get(first_product_url)  # 打开产品页面\n",
    "    sleep(3)\n",
    "\n",
    "    # 切换到评论标签\n",
    "    comment_tab = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'li[data-anchor=\"#comment\"]')))\n",
    "    comment_tab.click()\n",
    "    sleep(3)\n",
    "\n",
    "    # 定义函数以提取评论内容\n",
    "    def extract_comments():\n",
    "        global comments  # 使用全局变量来存储评论数据\n",
    "        comment_items = driver.find_elements(By.CSS_SELECTOR, 'div.comment-item')\n",
    "        for comment in comment_items:\n",
    "            C = {}\n",
    "            C['content'] = comment.find_element(By.CSS_SELECTOR, 'p.comment-con').text  # 提取评论内容\n",
    "            C['time'] = comment.find_element(By.CSS_SELECTOR, 'span.comment-time').text  # 提取评论时间\n",
    "            C['user'] = comment.find_element(By.CSS_SELECTOR, 'div.user-info').text  # 提取用户名\n",
    "            C['score'] = len(comment.find_elements(By.CSS_SELECTOR, 'span.star-red'))  # 提取评分\n",
    "            comments = pd.concat([comments, pd.DataFrame([C])])\n",
    "\n",
    "    # 定义函数以点击下一页的评论按钮\n",
    "    def next_comment_page():\n",
    "        try:\n",
    "            next_button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'a.ui-pager-next')))\n",
    "            if 'disabled' not in next_button.get_attribute('class'):\n",
    "                next_button.click()\n",
    "                sleep(3)\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "        except:\n",
    "            return False\n",
    "\n",
    "    # 抓取指定页数的评论\n",
    "    for _ in range(comment_pages):\n",
    "        extract_comments()  # 提取评论\n",
    "        if not next_comment_page():  # 如果无法点击下一页评论按钮，停止抓取\n",
    "            break\n",
    "\n",
    "# 关闭浏览器\n",
    "driver.quit()\n",
    "\n",
    "# 打印产品和评论数据\n",
    "print(\"Products:\")\n",
    "print(data)\n",
    "print(\"\\nComments:\")\n",
    "print(comments)\n",
    "\n",
    "# 保存产品和评论数据到 CSV 文件\n",
    "data.to_csv('jd_products.csv', index=False, encoding='utf-8-sig')\n",
    "comments.to_csv('jd_comments.csv', index=False, encoding='utf-8-sig')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9931b1b0-0a3b-4fa4-83d0-1817bc12f3c7",
   "metadata": {},
   "source": [
    "以上代码虽然爬取到了商品 但是却没有拿到评论 原因是jd认为我们是在做爬虫 下面我们通过设置options = uc.ChromeOptions()\n",
    "    options.add_argument(\"--disable-gpu\")\n",
    "    options.add_argument(\"--no-sandbox\")\n",
    "    options.add_argument(\"--disable-extensions\")\n",
    "    options.add_argument(\"--window-size=1920,1080\")\n",
    "    来绕过爬虫 拿到评论信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9a827082-2215-42fb-93af-beaf731f9861",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-22 03:25:10,293 - INFO - ====== WebDriver manager ======\n",
      "2024-10-22 03:25:11,778 - INFO - Get LATEST chromedriver version for google-chrome\n",
      "2024-10-22 03:25:11,961 - INFO - Get LATEST chromedriver version for google-chrome\n",
      "2024-10-22 03:25:12,132 - INFO - Driver [C:\\Users\\Administrator\\.wdm\\drivers\\chromedriver\\win64\\130.0.6723.58\\chromedriver-win32/chromedriver.exe] found in cache\n",
      "2024-10-22 03:25:13,448 - INFO - 成功创建 Chrome 驱动\n",
      "2024-10-22 03:25:15,086 - INFO - 已打开商品页面: https://item.jd.com/100095516761.html\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请在浏览器中登录，然后按回车键继续... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-22 03:25:42,940 - INFO - 用户确认已登录\n",
      "2024-10-22 03:25:42,995 - INFO - 已点击评论标签，使用选择器: li[data-anchor=\"#comment\"]\n",
      "2024-10-22 03:25:50,002 - INFO - 正在爬取第 1 页评论\n",
      "2024-10-22 03:25:50,030 - INFO - 找到评论元素，使用选择器: div.comment\n",
      "2024-10-22 03:25:50,039 - ERROR - 提取单条评论时出错: Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"p.comment-con, div.comment-content\"}\n",
      "  (Session info: chrome=130.0.6723.59); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\n",
      "Stacktrace:\n",
      "\tGetHandleVerifier [0x003674D3+23139]\n",
      "\t(No symbol) [0x002EFCA4]\n",
      "\t(No symbol) [0x001CC2A3]\n",
      "\t(No symbol) [0x0020DC66]\n",
      "\t(No symbol) [0x0020DEAB]\n",
      "\t(No symbol) [0x00203CD1]\n",
      "\t(No symbol) [0x0022FEC4]\n",
      "\t(No symbol) [0x00203BF4]\n",
      "\t(No symbol) [0x00230114]\n",
      "\t(No symbol) [0x00249564]\n",
      "\t(No symbol) [0x0022FC16]\n",
      "\t(No symbol) [0x0020216C]\n",
      "\t(No symbol) [0x002030FD]\n",
      "\tGetHandleVerifier [0x005FBAA3+2727987]\n",
      "\tGetHandleVerifier [0x0065476E+3091710]\n",
      "\tGetHandleVerifier [0x0064CF52+3060962]\n",
      "\tGetHandleVerifier [0x00401460+653808]\n",
      "\t(No symbol) [0x002F8E5D]\n",
      "\t(No symbol) [0x002F5A48]\n",
      "\t(No symbol) [0x002F5BE0]\n",
      "\t(No symbol) [0x002E8110]\n",
      "\tBaseThreadInitThunk [0x75CD7BA9+25]\n",
      "\tRtlInitializeExceptionChain [0x76F7C0CB+107]\n",
      "\tRtlClearBits [0x76F7C04F+191]\n",
      "\n",
      "2024-10-22 03:25:50,040 - INFO - 已提取 1 条评论\n",
      "2024-10-22 03:26:50,928 - ERROR - 无法找到下一页按钮\n",
      "2024-10-22 03:26:53,349 - INFO - 已关闭浏览器\n",
      "2024-10-22 03:26:53,350 - WARNING - 未成功爬取到评论\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium_stealth import stealth\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException, WebDriverException, NoSuchElementException\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import random\n",
    "import logging\n",
    "import traceback\n",
    "\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n",
    "\n",
    "def setup_driver():\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.add_argument(\"--disable-gpu\")\n",
    "    options.add_argument(\"--no-sandbox\")\n",
    "    options.add_argument(\"--disable-extensions\")\n",
    "    options.add_argument(\"--window-size=1920,1080\")\n",
    "    options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
    "    options.add_experimental_option('useAutomationExtension', False)\n",
    "    \n",
    "    try:\n",
    "        service = Service(ChromeDriverManager().install())\n",
    "        driver = webdriver.Chrome(service=service, options=options)\n",
    "        \n",
    "        stealth(driver,\n",
    "                languages=[\"zh-CN\", \"zh\"],\n",
    "                vendor=\"Google Inc.\",\n",
    "                platform=\"Win32\",\n",
    "                webgl_vendor=\"Intel Inc.\",\n",
    "                renderer=\"Intel Iris OpenGL Engine\",\n",
    "                fix_hairline=True,\n",
    "        )\n",
    "        \n",
    "        logging.info(\"成功创建 Chrome 驱动\")\n",
    "        return driver\n",
    "    except Exception as e:\n",
    "        logging.error(f\"创建驱动失败: {e}\")\n",
    "        logging.error(traceback.format_exc())\n",
    "        raise\n",
    "\n",
    "def open_product_page(driver, product_id):\n",
    "    url = f\"https://item.jd.com/{product_id}.html\"\n",
    "    try:\n",
    "        driver.get(url)\n",
    "        logging.info(f\"已打开商品页面: {url}\")\n",
    "        time.sleep(random.uniform(3, 5))\n",
    "    except Exception as e:\n",
    "        logging.error(f\"打开商品页面失败: {e}\")\n",
    "        raise\n",
    "\n",
    "def wait_for_login(driver):\n",
    "    input(\"请在浏览器中登录，然后按回车键继续...\")\n",
    "    logging.info(\"用户确认已登录\")\n",
    "\n",
    "def navigate_to_comments(driver, wait):\n",
    "    try:\n",
    "        # 尝试多个可能的选择器\n",
    "        selectors = [\n",
    "            'li[data-anchor=\"#comment\"]',\n",
    "            'a[href=\"#comment\"]',\n",
    "            'div[id=\"comment\"]',\n",
    "            'div.tab-main:nth-child(2)'\n",
    "        ]\n",
    "        \n",
    "        for selector in selectors:\n",
    "            try:\n",
    "                comment_tab = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, selector)))\n",
    "                comment_tab.click()\n",
    "                logging.info(f\"已点击评论标签，使用选择器: {selector}\")\n",
    "                break\n",
    "            except:\n",
    "                continue\n",
    "        else:\n",
    "            raise Exception(\"无法找到评论标签\")\n",
    "        \n",
    "        time.sleep(5)\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        time.sleep(2)\n",
    "        \n",
    "    except Exception as e:\n",
    "        logging.error(f\"导航到评论区时出错: {e}\")\n",
    "        raise\n",
    "\n",
    "def save_page_source(driver):\n",
    "    try:\n",
    "        with open(\"jd_comment_page.html\", \"w\", encoding=\"utf-8\") as f:\n",
    "            f.write(driver.page_source)\n",
    "        logging.info(\"已保存页面源代码到 jd_comment_page.html\")\n",
    "    except Exception as e:\n",
    "        logging.error(f\"保存页面源代码失败: {e}\")\n",
    "\n",
    "def extract_comments(driver, wait, comments):\n",
    "    try:\n",
    "        # 尝试多个可能的选择器\n",
    "        selectors = [\n",
    "            'div.comment-item',\n",
    "            'div.comment-list div',\n",
    "            'div[data-guid]',\n",
    "            'div.comment'\n",
    "        ]\n",
    "        \n",
    "        comment_items = []\n",
    "        for selector in selectors:\n",
    "            comment_items = driver.find_elements(By.CSS_SELECTOR, selector)\n",
    "            if comment_items:\n",
    "                logging.info(f\"找到评论元素，使用选择器: {selector}\")\n",
    "                break\n",
    "        \n",
    "        if not comment_items:\n",
    "            logging.warning(\"未找到评论元素，可能页面结构已改变\")\n",
    "            save_page_source(driver)\n",
    "            return\n",
    "        \n",
    "        for item in comment_items:\n",
    "            try:\n",
    "                content = item.find_element(By.CSS_SELECTOR, 'p.comment-con, div.comment-content').text\n",
    "                time_elem = item.find_element(By.CSS_SELECTOR, 'span.comment-time, span.time').text\n",
    "                user = item.find_element(By.CSS_SELECTOR, 'div.user-info, span.user-name').text\n",
    "                stars = len(item.find_elements(By.CSS_SELECTOR, 'span.star-red, i.star-yellow'))\n",
    "                \n",
    "                comment = {\n",
    "                    'content': content.strip() if content else \"N/A\",\n",
    "                    'time': time_elem.strip() if time_elem else \"N/A\",\n",
    "                    'user': user.strip() if user else \"N/A\",\n",
    "                    'score': stars if stars else 0\n",
    "                }\n",
    "                comments.append(comment)\n",
    "            except Exception as e:\n",
    "                logging.error(f\"提取单条评论时出错: {e}\")\n",
    "        \n",
    "        logging.info(f\"已提取 {len(comment_items)} 条评论\")\n",
    "    except Exception as e:\n",
    "        logging.error(f\"提取评论过程中发生错误: {e}\")\n",
    "        raise\n",
    "\n",
    "def next_page(driver, wait):\n",
    "    try:\n",
    "        # 尝试多个可能的选择器\n",
    "        selectors = [\n",
    "            'a.ui-pager-next',\n",
    "            'a.next',\n",
    "            'a[aria-label=\"Next\"]'\n",
    "        ]\n",
    "        \n",
    "        for selector in selectors:\n",
    "            try:\n",
    "                next_button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, selector)))\n",
    "                if 'disabled' not in next_button.get_attribute('class'):\n",
    "                    next_button.click()\n",
    "                    logging.info(f\"已点击下一页，使用选择器: {selector}\")\n",
    "                    time.sleep(random.uniform(3, 5))\n",
    "                    return True\n",
    "                else:\n",
    "                    logging.info(\"已到达最后一页\")\n",
    "                    return False\n",
    "            except:\n",
    "                continue\n",
    "        \n",
    "        logging.error(\"无法找到下一页按钮\")\n",
    "        return False\n",
    "    except Exception as e:\n",
    "        logging.error(f\"翻页时发生错误: {e}\")\n",
    "        return False\n",
    "\n",
    "def main():\n",
    "    product_id = '100095516761'  # 更换为您想爬取的商品ID\n",
    "    comment_pages = 5\n",
    "    comments = []\n",
    "    driver = None\n",
    "\n",
    "    try:\n",
    "        driver = setup_driver()\n",
    "        wait = WebDriverWait(driver, 20)\n",
    "\n",
    "        open_product_page(driver, product_id)\n",
    "        wait_for_login(driver)  # 等待登录确认\n",
    "        navigate_to_comments(driver, wait)\n",
    "        \n",
    "        for page in range(comment_pages):\n",
    "            logging.info(f\"正在爬取第 {page + 1} 页评论\")\n",
    "            extract_comments(driver, wait, comments)\n",
    "            if not next_page(driver, wait):\n",
    "                break\n",
    "\n",
    "    except Exception as e:\n",
    "        logging.error(f\"爬取过程中发生错误: {e}\")\n",
    "        logging.error(traceback.format_exc())\n",
    "    finally:\n",
    "        if driver:\n",
    "            driver.quit()\n",
    "            logging.info(\"已关闭浏览器\")\n",
    "\n",
    "    if comments:\n",
    "        comments_df = pd.DataFrame(comments)\n",
    "        comments_df.to_csv('jd_comments.csv', index=False, encoding='utf-8-sig')\n",
    "        logging.info(f\"成功爬取 {len(comments_df)} 条评论，已保存到 jd_comments.csv 文件\")\n",
    "        logging.info(\"\\n评论示例:\")\n",
    "        logging.info(comments_df.head().to_string())\n",
    "    else:\n",
    "        logging.warning(\"未成功爬取到评论\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7b0dc46-bcdd-4b5d-b099-57f9c343e012",
   "metadata": {},
   "source": [
    "我们主要通过以下几种方式来绕过反爬虫机制：\n",
    "\n",
    "1. 使用 undetected_chromedriver：\n",
    "   这是最关键的一步。undetected_chromedriver 是一个专门设计用来绕过网站对 Selenium 自动化的检测的工具。它修改了 ChromeDriver 的特征，使其更难被识别为自动化工具。\n",
    "\n",
    "2. 模拟真实用户行为：\n",
    "   - 添加随机延时：使用 `random.uniform()` 在操作之间添加随机时间间隔，模拟人类操作的不规则性。\n",
    "   - 滚动页面：使用 `driver.execute_script()` 执行滚动操作，模拟用户浏览页面的行为。\n",
    "\n",
    "3. 自定义 Chrome 选项：\n",
    "   - 禁用 GPU 加速（`--disable-gpu`）\n",
    "   - 禁用沙盒模式（`--no-sandbox`）\n",
    "   - 禁用扩展（`--disable-extensions`）\n",
    "   - 设置窗口大小（`--window-size=1920,1080`）\n",
    "   这些设置可以帮助浏览器看起来更像真实用户的浏览器。\n",
    "\n",
    "4. 使用真实的 User-Agent：\n",
    "   虽然在这个脚本中没有明确设置，但 undetected_chromedriver 通常会使用一个真实的 User-Agent。\n",
    "\n",
    "5. 分步加载和等待：\n",
    "   在导航到评论页面后，我们添加了额外的等待时间和滚动操作，确保页面完全加载。\n",
    "\n",
    "6. 错误处理和重试机制：\n",
    "   虽然这个脚本中没有明确的重试逻辑，但我们添加了大量的错误处理，这有助于脚本在遇到临时问题时能够继续运行。\n",
    "\n",
    "7. 避免频繁请求：\n",
    "   通过限制爬取的页数（在这个例子中是5页），我们减少了被检测为机器人的可能性。\n",
    "\n",
    "8. 保存页面源代码：\n",
    "   虽然这不直接帮助绕过反爬虫，但它允许我们在遇到问题时分析页面结构，有助于调整爬虫策略。\n",
    "\n",
    "这些方法综合起来，使得我们的爬虫行为更接近于真实用户，从而降低了被检测和阻止的可能性。然而，需要注意的是，网站的反爬虫机制可能会不断更新，所以这种方法可能需要定期调整以保持有效。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3aa422c7-00ae-4186-a3b2-38384148c5a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting webdriver_manager\n",
      "  Downloading webdriver_manager-4.0.2-py2.py3-none-any.whl.metadata (12 kB)\n",
      "Requirement already satisfied: requests in d:\\anaconda3\\lib\\site-packages (from webdriver_manager) (2.32.2)\n",
      "Requirement already satisfied: python-dotenv in d:\\anaconda3\\lib\\site-packages (from webdriver_manager) (0.21.0)\n",
      "Requirement already satisfied: packaging in d:\\anaconda3\\lib\\site-packages (from webdriver_manager) (23.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in d:\\anaconda3\\lib\\site-packages (from requests->webdriver_manager) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in d:\\anaconda3\\lib\\site-packages (from requests->webdriver_manager) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in d:\\anaconda3\\lib\\site-packages (from requests->webdriver_manager) (2.0.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda3\\lib\\site-packages (from requests->webdriver_manager) (2024.8.30)\n",
      "Downloading webdriver_manager-4.0.2-py2.py3-none-any.whl (27 kB)\n",
      "Installing collected packages: webdriver_manager\n",
      "Successfully installed webdriver_manager-4.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install webdriver_manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0c6e73b-c3c0-4d37-ad31-6232f924e3c4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
